"""
Simplify (Flemish) CKAN JSON files to a format more suitable for data.gov.be.
"""

__author__ = "Bart Hanssens <bart.hanssens@fedict.be>"

import argparse, os, logging
import json

logging.basicConfig(filename='corve-simplify.log', level=logging.DEBUG)

def ckanparse(infile, outfile):
	"""Parse CKAN JSON file and store it in a new (simplified) JSON file."""
	logging.info('Parsing ' + str(infile))

	data = {} 

	with open(infile, 'r') as f:
		d = json.load(f)
		res = d['result']

		data['title'] = res['title']
		data['body'] = ''

		if 'author' in res and res['author']:
			data['source'] = res['author']

		for k in ('contexts', 'fmts', 'tags', 'license'):
			data[k] = set()	

		if 'notes' in res and res['notes']:
			data['body'] = '\n'.join((data['body'], res['notes']))

		if 'isopen' in res and res['isopen']:
			data['license'].add('--Gratis Open Data Licentie Vlaanderen')
		else:
			data['license'].add('Zie website van de data leverancier')

		if 'organization' in res:
			if res['organization'] and res['organization']['name']:
				data['org'] = res['organization']['name']

		if 'name' in res:
			data['contexts'].add('http://ckan-001.corve.openminds.be/dataset/' + res['name'])
	
		for r in res['resources']:
			if r['format']:
				data['fmts'].add(r['format'])
			if r['name']:
				data['body'] = '\n'.join((data['body'], r['name'])) 

		for e in res['extras']:
			if e['value']:
				if e['key'] == 'beleidsdomein':
					data['tags'].add(e['value'])
				if e['key'] == 'dekking in tijd':
					data['time'] = e['value']
				if e['key'] == 'frequency-of-update':
					data['freq'] = e['value']
				if e['key'] == 'geografische dekking':
					data['geo'] = e['value']
				if e['key'] == 'licence':
					data['body'] = '\n'.join((data['body'], e['value']))
				if e['key'] == 'access_constraints':
					data['body'] = '\n'.join((data['body'], e['value']))

	# Copy formats to media, for additional mapping
	data['media'] = list(data['fmts'])

	# Add defaults for required fields
	if not 'org' in data:
		data['org'] = 'andere'
	if not data['body']:
		data['body'] = data['title']
	data['body'] = data['body'].strip()

	if not data['tags']:
		data['tags'] = ['bestuurszaken - algemeen']

	# make entries JSON serializable 
	for k in ('contexts', 'tags', 'fmts', 'license'): 
		data[k] = list(data[k])
	
	with open(outfile, 'w') as f:
		json.dump(data, f, indent=4)


def main():	
	parser = argparse.ArgumentParser(description='Simplify Flemish CKAN JSON files')
	parser.add_argument('--indir', help='Input directory', required=True)
	parser.add_argument('--outdir', help='Output directory', required=True)
	
	args = parser.parse_args()
	
	for f in os.listdir(args.indir):
		ckanparse(os.path.join(args.indir, f), os.path.join(args.outdir, f))
			
if __name__ == "__main__":
	main()
